/************************************************************
 * DS_TOKENIZER.H                                  10.11.2006
 *
 * .
 ************************************************************/
#ifndef _DS_TOKENIZER_H_
#define _DS_TOKENIZER_H_

//#include "ds_strid.h"

namespace ds {

//////////////////////////////////////////////////////////////////////////
// TOKENIZER

enum TOKENIZE_RESULT {
   TOKENIZE_CONTINUE,
   TOKENIZE_BREAK,
   TOKENIZE_ERROR
};

// Token processor must be like DEF_TOKEN_PROCESSOR
class DEF_TOKEN_PROCESSOR {
public:
   DEF_TOKEN_PROCESSOR(dsVECTOR<dsSTRING> & tokens)
      : tokens(tokens)
   {}

   // String parsing is stopped if any of these methods returns PARSERES_BREAK or PARSERES_ERROR
   TOKENIZE_RESULT            ProcessToken         (const char * token, int length)
   { if (length > 0) tokens.PushBack(token); return TOKENIZE_CONTINUE; }

   TOKENIZE_RESULT            ProcessDelim         (char c)
   { return TOKENIZE_CONTINUE; }

private:
   dsVECTOR<dsSTRING> &       tokens;
};

template <class PROCESSOR_T>
TOKENIZE_RESULT   Tokenize       (const dsSTRING & str, const dsSTRING & delims, PROCESSOR_T & tokenizer);

inline void       Tokenize       (const dsSTRING & str, const dsSTRING & delims, dsVECTOR<dsSTRING> & result)
{ DEF_TOKEN_PROCESSOR processor(result); Tokenize(str, delims, processor); }

//////////////////////////////////////////////////////////////////////////
// CPP-line tokenizer
enum TOKEN_TYPE {
   TOKEN_EOF = -1,
   TOKEN_WORD,             // abc_000
   TOKEN_STRING,           // "string"
   TOKEN_INT,              // [-|+][ ]*d+ (e.g. - 12345)
   TOKEN_FLOAT,            // [-|+][ ]*d+[.]d+
   //TOKEN_CHAR,             // 'c'
   // TODO: support escape sequences for string literals
   TOKEN_USER,
};

struct TOKEN {
   int         token;
   int         line;
   dsSTRING    value;

   TOKEN(void)
      : token(TOKEN_EOF)
      , line(0)
   {}
};

enum TOKENIZER_ERROR_TYPE {
   TOKEN_ERR_UNEXPECTED_SYMBOL,
   TOKEN_ERR_EOF_IN_COMMENT,
   TOKEN_ERR_EOLN_IN_LITERAL,
};

class TOKENIZER_ERROR_CALLBACK {
public:
   virtual void           Error          (TOKENIZER_ERROR_TYPE err, int line, char ch = ' ') = 0;

   virtual ~TOKENIZER_ERROR_CALLBACK(void) {}
};

void TokenizeCppLike    (const dsSTRING & input, dsVECTOR<TOKEN> & tokens); // deprecated, use CPP_LIKE_TOKENIZER class instead

class CPP_LIKE_TOKENIZER {
public:
   void                 RegisterToken        (int type, char symbol); // single-symbol token
   void                 RegisterCppSymbols   (int type);
   void                 DisableToken         (int type, bool disable = true); // disable registered token

   bool                 Tokenize             (dsSTRING text, dsVECTOR<TOKEN> & res, TOKENIZER_ERROR_CALLBACK * err = NULL);

   CPP_LIKE_TOKENIZER(void)
      : disabledTokens(0)
   {}

private:
   static bool          ParseComments        (dsSTRING text, int & index, int & line, TOKENIZER_ERROR_CALLBACK * err);

private:
   struct USER_TOKEN {
      int               token;
      dsSTRING          pattern;

      bool operator < (const USER_TOKEN & p) const { return pattern < p.pattern; }
   };

   dsVECTOR<USER_TOKEN> tokens;
   int                  disabledTokens;
};

/*************************************************************************/
/* ds::IsSymbol 
/*
/*************************************************************************/
inline bool IsSymbol(char ch)
{
   switch (ch) {
      case '+': case '-': case '*': case '/': case '%':
      case '|': case '&': case '^': case '~':
      case '(': case ')': case '[': case ']':
      case '<': case '>': case '!': case '=':
      case ',': case '.':
         return true;

   default:
      return false;
   }
}

/*************************************************************************/
/* ds::IsSpace 
/*
/*************************************************************************/
inline bool IsSpace(char ch)
{
   return ch <= ' ';
}

/*************************************************************************/
/* ds::IsDigit 
/*
/*************************************************************************/
inline bool IsDigit(char ch)
{
   return ch >= '0' && ch <= '9';
}

/*************************************************************************/
/* ds::IsAlpha 
/*
/*************************************************************************/
inline bool IsAlpha(char ch)
{
   return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z';
}

/*************************************************************************/
/* ds::IsWordLetter 
/*
/*************************************************************************/
inline bool IsWordLetter(char ch, bool isFirst = false)
{
   if (ch == '_' || IsAlpha(ch)) {
      return true;
   }
   if (!isFirst && IsDigit(ch)) {
      return true;
   }
   return false;
}

extern const dsSTRING CPP_SYMBOLS;

int GetIndent(const dsSTRING & text);
int GetIndent(const dsSTRING & text, dsVECTOR<dsSTRING> & lines);
dsSTRING BuildStringFromLines(const dsVECTOR<dsSTRING> & lines, int start = 0, int count = -1);
dsSTRING SetIndent(const dsSTRING & text, int indent);

#include "ds_tokenizer.hpp"

} // namespace ds

#endif // _DS_TOKENIZER_H_
